package demo3_5;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class SplitData_2 {
	private static long train_male = 0; //训练男
	private static long train_female = 0; //训练女
	private static long test_male = 0; // 测试男
	private static long test_female = 0; //测试女
	private static long validate_male = 0; //验证男
	private static long validate_female = 0; //验证女
	
	public static void main(String[] args) throws Exception {
		Configuration conf=new Configuration();
		conf.set("fs.defaultFS", "DragonBoy:9000");
		FileSystem fs=FileSystem.get(conf);
		//获取预处理之后的电影数据路径
		Path moviedata=new Path("/movie/processing_out/part-m-00000");
		//得到电影数据大小
		int datasize=getSize(fs, moviedata);
		System.out.println("总记录数:"+datasize);
		
		//训练
		Path train=new Path("hdfs://DragonBoy:9000/movie/trainData");
		fs.delete(train,true);
		FSDataOutputStream os1=fs.create(train);
		BufferedWriter bw1=new BufferedWriter(new OutputStreamWriter(os1));
		//测试
		Path test=new Path("hdfs://DragonBoy:9000/movie/testData");
		fs.delete(test,true);
		FSDataOutputStream os2=fs.create(test);
		BufferedWriter bw2=new BufferedWriter(new OutputStreamWriter(os2));
		//验证
		Path validate=new Path("hdfs://DragonBoy:9000/movie/validateData");
		fs.delete(validate,true);
		FSDataOutputStream os3=fs.create(validate);
		BufferedWriter bw3=new BufferedWriter(new OutputStreamWriter(os3));
		//
		FSDataInputStream is=fs.open(moviedata);
		BufferedReader br=new BufferedReader(new InputStreamReader(is));
		
//		//训练
//		Set<Integer> train_index= new HashSet<Integer>();
//		
//		//验证
//		Set<Integer> validate_index=  new HashSet<Integer>();
		String line="";
		String subLine="";
		
//		int sum=0;
		int trainsize=0;
		int testsize=0;
		int validatesize=0;
		int train_MaxSize = (int)(datasize*0.8);
		int test_MaxSize = (int)(datasize*0.1);
		int validate_MaxSize =(int)(datasize*0.1);
		
		while((line=br.readLine())!=null){	
			//1,1,1,10,48067,5,5,18,20,14,2,0,21,3,0,0,14,0,6,3,3,2,0
			subLine = line.substring(line.indexOf(",")+1,line.indexOf(",")+2);//判断男女		
			if(subLine.equals("0")) {
				double number = Math.random();
				if(number<0.33 && validatesize<=validate_MaxSize) {
					//划给验证
					bw3.write(line.toString());
					bw3.newLine();
					
					validate_male++; //验证男 人数
					validatesize++;// 当前验证 总人数
					
				}else if(number<0.66 && testsize<=test_MaxSize) {
					//划给测试
					bw2.write(line.toString());
					bw2.newLine();
					
					test_male++; //测试男 人数
					testsize++; //当前测试总人数
					
				}else {
					//划给训练
					bw1.write(line.toString());
					bw1.newLine();
					
					
					train_male++;//训练男 人数
					trainsize++; //当前训练 总人数
				}
				
				
			}else {
				if(0.09>Math.random() && testsize<=test_MaxSize) { //1709的 0.09 约等于 150人左右
					//未达到测试总人数
					
					//9%的女性数据给测试，91%的女性数据给训练
					bw2.write(line.toString());
					bw2.newLine();
					
					test_female++; // 测试 女人数
					
					testsize++; //当前测试总人数
					
					
				}else {
					
					bw1.write(line.toString());
					bw1.newLine();
					
					train_female++; //训练女 人数
					trainsize++;//当前训练 总人数
				}
				
			}
//			sum+=1;
			
		}//while	
		System.out.println("训练数据总记录数:"+trainsize);
		
		
		System.out.println("训练里的男数据："+train_male);
		System.out.println("训练里的女数据："+train_female);
		
		System.out.println("测试数据总记录数:"+testsize);
		
		System.out.println("测试里的男数据："+test_male);
		System.out.println("测试里的女数据："+test_female);
		
		System.out.println("验证数据总记录数:"+validatesize);
		
		System.out.println("验证里的男数据："+validate_male);
		System.out.println("验证里的女数据："+validate_female);
		
		bw1.close();
		os1.close();
		bw2.close();
		os2.close();
		bw3.close();
		os3.close();
		br.close();
		is.close();
		fs.close();
	}
	/**
	 * 读取原始数据并统计数据的记录数
	 * @param fs
	 * @param path
	 * @return
	 * @throws Exception
	 */
	public static int getSize(FileSystem fs,Path path) throws Exception{
		int count=0;
		FSDataInputStream is=fs.open(path);
		BufferedReader br=new BufferedReader(new InputStreamReader(is));
		String line="";
		while((line=br.readLine())!=null){
			count++;
		}
		br.close();
		is.close();
		return count;		
	}
	/**
	 *随机获取80%的原始数据下标 -训练数据
	 * @param count
	 * @return
	 */
//	public static Set<Integer> trainIndex(int count){
//		Set<Integer> train_index=new HashSet<Integer>();
//		int trainSplitNum=(int)(count*0.8);
//		Random random=new Random();
//		while(train_index.size()<trainSplitNum){
//			int a=random.nextInt(count);
//			train_index.add(a);
//		}
//		return train_index;	
//	}
	/**
	 * 随机获取10%的原始数据下标-验证数据
	 * @param count
	 * @param train_index
	 * @return
	 */
//	public static Set<Integer> validateIndex(int count,Set<Integer> train_index){
//		Set<Integer> validate_index=new HashSet<Integer>();
//		int validateSplitNum=count-(int)(count*0.9);
//		Random random=new Random();
//		while(validate_index.size()<validateSplitNum){
//			int a=random.nextInt(count);
//			if(!train_index.contains(a)){
//				validate_index.add(a);	
//			}
//		}
//		return validate_index;	
//	}
	
}
